
<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
      
      
        <link rel="canonical" href="https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/text_utils.html">
      
      
        <link rel="prev" href="image_utils.html">
      
      
        <link rel="next" href="../preprocessing.html">
      
      
      <link rel="icon" href="../../assets/images/favicon.ico">
      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
    
    
      
        <title>Text utils - pytorch_widedeep</title>
      
    
    
      <link rel="stylesheet" href="../../assets/stylesheets/main.0253249f.min.css">
      
        
        <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
      
      


    
    
      
    
    
      
        
        
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
    
      <link rel="stylesheet" href="../../assets/_mkdocstrings.css">
    
      <link rel="stylesheet" href="../../stylesheets/extra.css">
    
    <script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      

    
    
    
  </head>
  
  
    
    
      
    
    
    
    
    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="red" data-md-color-accent="deep-orange">
  
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#text-utils" class="md-skip">
          Skip to content
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
    
      

  

<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href="../../index.html" title="pytorch_widedeep" class="md-header__button md-logo" aria-label="pytorch_widedeep" data-md-component="logo">
      
  <img src="../../assets/images/widedeep_logo.png" alt="logo">

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            pytorch_widedeep
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              Text utils
            
          </span>
        </div>
      </div>
    </div>
    
      
        <form class="md-header__option" data-md-component="palette">
  
    
    
    
    <input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="red" data-md-color-accent="deep-orange"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_0">
    
      <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
      </label>
    
  
    
    
    
    <input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="red" data-md-color-accent="deep-orange"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_1">
    
      <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
      </label>
    
  
</form>
      
    
    
      <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
    
    
    
      <label class="md-header__button md-icon" for="__search">
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
      </label>
      <div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
      <label class="md-search__icon md-icon" for="__search">
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
      </label>
      <nav class="md-search__options" aria-label="Search">
        
        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
          
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
        </button>
      </nav>
      
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
        <div class="md-search-result" data-md-component="search-result">
          <div class="md-search-result__meta">
            Initializing search
          </div>
          <ol class="md-search-result__list" role="presentation"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
    
    
      <div class="md-header__source">
        <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
  <div class="md-source__icon md-icon">
    
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
  </div>
  <div class="md-source__repository">
    pytorch_widedeep
  </div>
</a>
      </div>
    
  </nav>
  
    
      
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
  <div class="md-grid">
    <ul class="md-tabs__list">
      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../../index.html" class="md-tabs__link">
        
  
    
  
  Home

      </a>
    </li>
  

      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../../installation.html" class="md-tabs__link">
        
  
    
  
  Installation

      </a>
    </li>
  

      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../../quick_start.html" class="md-tabs__link">
        
  
    
  
  Quick Start

      </a>
    </li>
  

      
        
  
  
    
  
  
    
    
      
  
  
    
  
  
    
    
      <li class="md-tabs__item md-tabs__item--active">
        <a href="index.html" class="md-tabs__link">
          
  
    
  
  Pytorch-widedeep

        </a>
      </li>
    
  

    
  

      
        
  
  
  
    
    
      <li class="md-tabs__item">
        <a href="../../examples/01_preprocessors_and_utils.html" class="md-tabs__link">
          
  
    
  
  Examples

        </a>
      </li>
    
  

      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../../contributing.html" class="md-tabs__link">
        
  
    
  
  Contributing

      </a>
    </li>
  

      
    </ul>
  </div>
</nav>
    
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
        
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    


  


  

<nav class="md-nav md-nav--primary md-nav--lifted md-nav--integrated" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="../../index.html" title="pytorch_widedeep" class="md-nav__button md-logo" aria-label="pytorch_widedeep" data-md-component="logo">
      
  <img src="../../assets/images/widedeep_logo.png" alt="logo">

    </a>
    pytorch_widedeep
  </label>
  
    <div class="md-nav__source">
      <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
  <div class="md-source__icon md-icon">
    
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
  </div>
  <div class="md-source__repository">
    pytorch_widedeep
  </div>
</a>
    </div>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../../index.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Home
  </span>
  

      </a>
    </li>
  

    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../../installation.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Installation
  </span>
  

      </a>
    </li>
  

    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../../quick_start.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Quick Start
  </span>
  

      </a>
    </li>
  

    
      
      
  
  
    
  
  
  
    
    
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
    
    
      
        
        
      
      
    
    
    <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
      
        
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
        
          
          <label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="">
            
  
  <span class="md-ellipsis">
    Pytorch-widedeep
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
          <label class="md-nav__title" for="__nav_4">
            <span class="md-nav__icon md-icon"></span>
            Pytorch-widedeep
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
                
  
  
    
  
  
  
    
    
      
        
          
        
      
        
      
        
      
        
      
        
      
    
    
      
      
    
    
    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
      
        
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_1" checked>
        
          
          
          <div class="md-nav__link md-nav__container">
            <a href="index.html" class="md-nav__link ">
              
  
  <span class="md-ellipsis">
    Utils
  </span>
  

            </a>
            
              
              <label class="md-nav__link " for="__nav_4_1" id="__nav_4_1_label" tabindex="0">
                <span class="md-nav__icon md-icon"></span>
              </label>
            
          </div>
        
        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_1_label" aria-expanded="true">
          <label class="md-nav__title" for="__nav_4_1">
            <span class="md-nav__icon md-icon"></span>
            Utils
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="deeptabular_utils.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Deeptabular utils
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="fastai_transforms.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Fastai transforms
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="image_utils.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Image utils
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
    
  
  
  
    <li class="md-nav__item md-nav__item--active">
      
      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
      
      
        
      
      
        <label class="md-nav__link md-nav__link--active" for="__toc">
          
  
  <span class="md-ellipsis">
    Text utils
  </span>
  

          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <a href="text_utils.html" class="md-nav__link md-nav__link--active">
        
  
  <span class="md-ellipsis">
    Text utils
  </span>
  

      </a>
      
        

<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
  
    
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.utils.text_utils.simple_preprocess" class="md-nav__link">
    <span class="md-ellipsis">
      simple_preprocess
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.utils.text_utils.get_texts" class="md-nav__link">
    <span class="md-ellipsis">
      get_texts
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.utils.text_utils.pad_sequences" class="md-nav__link">
    <span class="md-ellipsis">
      pad_sequences
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="md-nav__link">
    <span class="md-ellipsis">
      build_embeddings_matrix
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
      
    </li>
  

              
            
          </ul>
        </nav>
      
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../preprocessing.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Preprocessing
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../load_from_folder.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Load From Folder
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../model_components.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Model Components
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../the_rec_module.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    The Rec Module
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../bayesian_models.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Bayesian models
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../losses.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Losses
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../metrics.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Metrics
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../dataloaders.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Dataloaders
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../callbacks.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Callbacks
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../trainer.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Trainer
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../bayesian_trainer.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Bayesian Trainer
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../self_supervised_pretraining.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Self Supervised Pretraining
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../tab2vec.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Tab2Vec
  </span>
  

      </a>
    </li>
  

              
            
          </ul>
        </nav>
      
    </li>
  

    
      
      
  
  
  
  
    
    
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
    
    
      
      
    
    
    <li class="md-nav__item md-nav__item--nested">
      
        
        
          
        
        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_5" >
        
          
          <label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    Examples
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_5">
            <span class="md-nav__icon md-icon"></span>
            Examples
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/01_preprocessors_and_utils.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    01_preprocessors_and_utils
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/02_model_components.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    02_model_components
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/03_binary_classification_with_defaults.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    03_binary_classification_with_defaults
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/04_regression_with_images_and_text.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    04_regression_with_images_and_text
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/05_save_and_load_model_and_artifacts.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    05_save_and_load_model_and_artifacts
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/06_finetune_and_warmup.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    06_finetune_and_warmup
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/07_custom_components.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    07_custom_components
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/08_custom_dataLoader_imbalanced_dataset.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    08_custom_dataLoader_imbalanced_dataset
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/09_extracting_embeddings.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    09_extracting_embeddings
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/10_3rd_party_integration-RayTune_WnB.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    10_3rd_party_integration-RayTune_WnB
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/11_auc_multiclass.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    11_auc_multiclass
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    12_ZILNLoss_origkeras_vs_pytorch_widedeep
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/13_model_uncertainty_prediction.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    13_model_uncertainty_prediction
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/14_bayesian_models.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    14_bayesian_models
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    15_Self-Supervised Pre-Training pt 1
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    15_Self-Supervised Pre-Training pt 2
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    16_Usign-a-custom-hugging-face-model
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    17_feature_importance_via_attention_weights
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    18_wide_and_deep_for_recsys_pt1
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    18_wide_and_deep_for_recsys_pt2
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/19_load_from_folder_functionality.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    19_load_from_folder_functionality
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    20-Using-huggingface-within-widedeep
  </span>
  

      </a>
    </li>
  

              
            
          </ul>
        </nav>
      
    </li>
  

    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../../contributing.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Contributing
  </span>
  

      </a>
    </li>
  

    
  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
          
          
            <div class="md-content" data-md-component="content">
              <article class="md-content__inner md-typeset">
                
                  

  
  


<h1 id="text-utils">Text utils<a class="headerlink" href="#text-utils" title="Permanent link">&para;</a></h1>
<p>Collection of helper function that facilitate processing text.</p>


<div class="doc doc-object doc-function">


<h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">simple_preprocess</span>


<a href="#pytorch_widedeep.utils.text_utils.simple_preprocess" class="headerlink" title="Permanent link">&para;</a></h2>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">simple_preprocess</span><span class="p">(</span>
    <span class="n">doc</span><span class="p">,</span> <span class="n">lower</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">deacc</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">max_len</span><span class="o">=</span><span class="mi">15</span>
<span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents first">

        <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to
indicate wether or not to lower case all the token in the doc</p>
<p>For more information see: <code>Gensim</code> <a href="https://radimrehurek.com/gensim/utils.html">utils module</a></p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>doc</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input document.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>lower</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Lower case tokens in the input doc</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>deacc</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>min_len</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p>
              </div>
            </td>
            <td>
                  <code>2</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>max_len</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p>
              </div>
            </td>
            <td>
                  <code>15</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">simple_preprocess</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">simple_preprocess</span><span class="p">(</span><span class="s1">&#39;Machine learning is great&#39;</span><span class="p">)</span>
<span class="go">[&#39;Machine&#39;, &#39;learning&#39;, &#39;is&#39;, &#39;great&#39;]</span>
</code></pre></div>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="typing.List">List</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List with the processed tokens</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">12</span>
<span class="normal">13</span>
<span class="normal">14</span>
<span class="normal">15</span>
<span class="normal">16</span>
<span class="normal">17</span>
<span class="normal">18</span>
<span class="normal">19</span>
<span class="normal">20</span>
<span class="normal">21</span>
<span class="normal">22</span>
<span class="normal">23</span>
<span class="normal">24</span>
<span class="normal">25</span>
<span class="normal">26</span>
<span class="normal">27</span>
<span class="normal">28</span>
<span class="normal">29</span>
<span class="normal">30</span>
<span class="normal">31</span>
<span class="normal">32</span>
<span class="normal">33</span>
<span class="normal">34</span>
<span class="normal">35</span>
<span class="normal">36</span>
<span class="normal">37</span>
<span class="normal">38</span>
<span class="normal">39</span>
<span class="normal">40</span>
<span class="normal">41</span>
<span class="normal">42</span>
<span class="normal">43</span>
<span class="normal">44</span>
<span class="normal">45</span>
<span class="normal">46</span>
<span class="normal">47</span>
<span class="normal">48</span>
<span class="normal">49</span>
<span class="normal">50</span>
<span class="normal">51</span>
<span class="normal">52</span>
<span class="normal">53</span>
<span class="normal">54</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">simple_preprocess</span><span class="p">(</span>
    <span class="n">doc</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
    <span class="n">lower</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
    <span class="n">deacc</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
    <span class="n">min_len</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
    <span class="n">max_len</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">15</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    This is `Gensim`&#39;s `simple_preprocess` with a `lower` param to</span>
<span class="sd">    indicate wether or not to lower case all the token in the doc</span>

<span class="sd">    For more information see: `Gensim` [utils module](https://radimrehurek.com/gensim/utils.html)</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    doc: str</span>
<span class="sd">        Input document.</span>
<span class="sd">    lower: bool, default = False</span>
<span class="sd">        Lower case tokens in the input doc</span>
<span class="sd">    deacc: bool, default = False</span>
<span class="sd">        Remove accent marks from tokens using `Gensim`&#39;s `deaccent`</span>
<span class="sd">    min_len: int, default = 2</span>
<span class="sd">        Minimum length of token (inclusive). Shorter tokens are discarded.</span>
<span class="sd">    max_len: int, default = 15</span>
<span class="sd">        Maximum length of token in result (inclusive). Longer tokens are discarded.</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess</span>
<span class="sd">    &gt;&gt;&gt; simple_preprocess(&#39;Machine learning is great&#39;)</span>
<span class="sd">    [&#39;Machine&#39;, &#39;learning&#39;, &#39;is&#39;, &#39;great&#39;]</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    List[str]</span>
<span class="sd">        List with the processed tokens</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">tokens</span> <span class="o">=</span> <span class="p">[</span>
        <span class="n">token</span>
        <span class="k">for</span> <span class="n">token</span> <span class="ow">in</span> <span class="n">tokenize</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="n">lower</span><span class="o">=</span><span class="n">lower</span><span class="p">,</span> <span class="n">deacc</span><span class="o">=</span><span class="n">deacc</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">&quot;ignore&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">min_len</span> <span class="o">&lt;=</span> <span class="nb">len</span><span class="p">(</span><span class="n">token</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="n">max_len</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">token</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;_&quot;</span><span class="p">)</span>
    <span class="p">]</span>
    <span class="k">return</span> <span class="n">tokens</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">get_texts</span>


<a href="#pytorch_widedeep.utils.text_utils.get_texts" class="headerlink" title="Permanent link">&para;</a></h2>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="n">already_processed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">n_cpus</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents first">

        <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a
series of very convenients things during the tokenization process</p>
<p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p>
<p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>:
<code>get_texts</code> uses <code>pytorch_widedeep.utils.fastai_transforms.Tokenizer</code>.
Such tokenizer uses a series of convenient processing steps, including
the  addition of some special tokens, such as <code>TK_MAJ</code> (<code>xxmaj</code>), used to
indicate the next word begins with a capital in the original text. For more
details of special tokens please see the <a href="https://docs.fast.ai/text.core.html#Tokenizing"><code>fastai</code> `docs</a></p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>texts</code>
            </td>
            <td>
                  <code><span title="typing.List">List</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of str with the texts (or documents). One str per document</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>already_processed</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[bool]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if the text is already processed and we simply want
to tokenize it. This parameter is thought for those cases where the
input sequences might not be text (but IDs, or anything else) and we
just want to tokenize it</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>n_cpus</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>number of CPUs to used during the tokenization process</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">get_texts</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;Machine learning is great&#39;</span><span class="p">,</span> <span class="s1">&#39;but building stuff is even better&#39;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
<span class="go">[[&#39;xxmaj&#39;, &#39;machine&#39;, &#39;learning&#39;, &#39;is&#39;, &#39;great&#39;], [&#39;but&#39;, &#39;building&#39;, &#39;stuff&#39;, &#39;is&#39;, &#39;even&#39;, &#39;better&#39;]]</span>
</code></pre></div>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="typing.List">List</span>[<span title="typing.List">List</span>[str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of lists, one list per '<em>document</em>' containing its corresponding tokens</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 57</span>
<span class="normal"> 58</span>
<span class="normal"> 59</span>
<span class="normal"> 60</span>
<span class="normal"> 61</span>
<span class="normal"> 62</span>
<span class="normal"> 63</span>
<span class="normal"> 64</span>
<span class="normal"> 65</span>
<span class="normal"> 66</span>
<span class="normal"> 67</span>
<span class="normal"> 68</span>
<span class="normal"> 69</span>
<span class="normal"> 70</span>
<span class="normal"> 71</span>
<span class="normal"> 72</span>
<span class="normal"> 73</span>
<span class="normal"> 74</span>
<span class="normal"> 75</span>
<span class="normal"> 76</span>
<span class="normal"> 77</span>
<span class="normal"> 78</span>
<span class="normal"> 79</span>
<span class="normal"> 80</span>
<span class="normal"> 81</span>
<span class="normal"> 82</span>
<span class="normal"> 83</span>
<span class="normal"> 84</span>
<span class="normal"> 85</span>
<span class="normal"> 86</span>
<span class="normal"> 87</span>
<span class="normal"> 88</span>
<span class="normal"> 89</span>
<span class="normal"> 90</span>
<span class="normal"> 91</span>
<span class="normal"> 92</span>
<span class="normal"> 93</span>
<span class="normal"> 94</span>
<span class="normal"> 95</span>
<span class="normal"> 96</span>
<span class="normal"> 97</span>
<span class="normal"> 98</span>
<span class="normal"> 99</span>
<span class="normal">100</span>
<span class="normal">101</span>
<span class="normal">102</span>
<span class="normal">103</span>
<span class="normal">104</span>
<span class="normal">105</span>
<span class="normal">106</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">get_texts</span><span class="p">(</span>
    <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
    <span class="n">already_processed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
    <span class="n">n_cpus</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Tokenization using `Fastai`&#39;s `Tokenizer` because it does a</span>
<span class="sd">    series of very convenients things during the tokenization process</span>

<span class="sd">    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`</span>

<span class="sd">    :information_source: **NOTE**:</span>
<span class="sd">    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</span>
<span class="sd">    Such tokenizer uses a series of convenient processing steps, including</span>
<span class="sd">    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</span>
<span class="sd">    indicate the next word begins with a capital in the original text. For more</span>
<span class="sd">    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    texts: List</span>
<span class="sd">        List of str with the texts (or documents). One str per document</span>
<span class="sd">    already_processed: bool, Optional, default = False</span>
<span class="sd">        Boolean indicating if the text is already processed and we simply want</span>
<span class="sd">        to tokenize it. This parameter is thought for those cases where the</span>
<span class="sd">        input sequences might not be text (but IDs, or anything else) and we</span>
<span class="sd">        just want to tokenize it</span>
<span class="sd">    n_cpus: int, Optional, default = None</span>
<span class="sd">        number of CPUs to used during the tokenization process</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.utils import get_texts</span>
<span class="sd">    &gt;&gt;&gt; texts = [&#39;Machine learning is great&#39;, &#39;but building stuff is even better&#39;]</span>
<span class="sd">    &gt;&gt;&gt; get_texts(texts)</span>
<span class="sd">    [[&#39;xxmaj&#39;, &#39;machine&#39;, &#39;learning&#39;, &#39;is&#39;, &#39;great&#39;], [&#39;but&#39;, &#39;building&#39;, &#39;stuff&#39;, &#39;is&#39;, &#39;even&#39;, &#39;better&#39;]]</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    List[List[str]]</span>
<span class="sd">        List of lists, one list per &#39;_document_&#39; containing its corresponding tokens</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">num_cpus</span> <span class="o">=</span> <span class="n">n_cpus</span> <span class="k">if</span> <span class="n">n_cpus</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>

    <span class="k">if</span> <span class="ow">not</span> <span class="n">already_processed</span><span class="p">:</span>
        <span class="n">processed_texts</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">simple_preprocess</span><span class="p">(</span><span class="n">t</span><span class="p">))</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">texts</span><span class="p">]</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">processed_texts</span> <span class="o">=</span> <span class="n">texts</span>
    <span class="n">tok</span> <span class="o">=</span> <span class="n">Tokenizer</span><span class="p">(</span><span class="n">n_cpus</span><span class="o">=</span><span class="n">num_cpus</span><span class="p">)</span><span class="o">.</span><span class="n">process_all</span><span class="p">(</span><span class="n">processed_texts</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">tok</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">pad_sequences</span>


<a href="#pytorch_widedeep.utils.text_utils.pad_sequences" class="headerlink" title="Permanent link">&para;</a></h2>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">pad_sequences</span><span class="p">(</span><span class="n">seq</span><span class="p">,</span> <span class="n">maxlen</span><span class="p">,</span> <span class="n">pad_first</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents first">

        <p>Given a List of tokenized and <code>numericalised</code> sequences it will return
padded sequences according to the input parameters.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>seq</code>
            </td>
            <td>
                  <code><span title="typing.List">List</span>[int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of int with the <code>numericalised</code> tokens</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>maxlen</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Maximum length of the padded sequences</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>pad_first</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Indicates whether the padding index will be added at the beginning or the
end of the sequences</p>
              </div>
            </td>
            <td>
                  <code>True</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>pad_idx</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.utils</span> <span class="kn">import</span> <span class="n">pad_sequences</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">seq</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pad_sequences</span><span class="p">(</span><span class="n">seq</span><span class="p">,</span> <span class="n">maxlen</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="go">array([0, 0, 1, 2, 3], dtype=int32)</span>
</code></pre></div>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>numpy array with the padded sequences</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">109</span>
<span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pad_sequences</span><span class="p">(</span>
    <span class="n">seq</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span> <span class="n">maxlen</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">pad_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">pad_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Given a List of tokenized and `numericalised` sequences it will return</span>
<span class="sd">    padded sequences according to the input parameters.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    seq: List</span>
<span class="sd">        List of int with the `numericalised` tokens</span>
<span class="sd">    maxlen: int</span>
<span class="sd">        Maximum length of the padded sequences</span>
<span class="sd">    pad_first: bool,  default = True</span>
<span class="sd">        Indicates whether the padding index will be added at the beginning or the</span>
<span class="sd">        end of the sequences</span>
<span class="sd">    pad_idx: int, default = 1</span>
<span class="sd">        padding index. Fastai&#39;s Tokenizer leaves 0 for the &#39;unknown&#39; token.</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences</span>
<span class="sd">    &gt;&gt;&gt; seq = [1,2,3]</span>
<span class="sd">    &gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)</span>
<span class="sd">    array([0, 0, 1, 2, 3], dtype=int32)</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        numpy array with the padded sequences</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">seq</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">maxlen</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s2">&quot;int32&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">pad_idx</span>
    <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">seq</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="n">maxlen</span><span class="p">:</span>
        <span class="n">res</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">seq</span><span class="p">[</span><span class="o">-</span><span class="n">maxlen</span><span class="p">:])</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;int32&quot;</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">res</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">res</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">maxlen</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s2">&quot;int32&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">pad_idx</span>
        <span class="k">if</span> <span class="n">pad_first</span><span class="p">:</span>
            <span class="n">res</span><span class="p">[</span><span class="o">-</span><span class="nb">len</span><span class="p">(</span><span class="n">seq</span><span class="p">)</span> <span class="p">:]</span> <span class="o">=</span> <span class="n">seq</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">res</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">seq</span><span class="p">)</span> <span class="p">:]</span> <span class="o">=</span> <span class="n">seq</span>
        <span class="k">return</span> <span class="n">res</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">build_embeddings_matrix</span>


<a href="#pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="headerlink" title="Permanent link">&para;</a></h2>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">build_embeddings_matrix</span><span class="p">(</span>
    <span class="n">vocab</span><span class="p">,</span> <span class="n">word_vectors_path</span><span class="p">,</span> <span class="n">min_freq</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span>
<span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents first">

        <p>Build the embedding matrix using pretrained word vectors.</p>
<p>Returns pretrained word embeddings. If a word in our vocabulary is not
among the pretrained embeddings it will be assigned the mean pretrained
word-embeddings vector</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>vocab</code>
            </td>
            <td>
                  <code><span title="typing.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a>, <span title="pytorch_widedeep.utils.fastai_transforms.ChunkVocab">ChunkVocab</span>]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>word_vectors_path</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>path to the pretrained word embeddings</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>min_freq</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>minimum frequency required for a word to be in the vocabulary</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>verbose</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>level of verbosity. Set to 0 for no verbosity</p>
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Pretrained word embeddings</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span>
<span class="normal">178</span>
<span class="normal">179</span>
<span class="normal">180</span>
<span class="normal">181</span>
<span class="normal">182</span>
<span class="normal">183</span>
<span class="normal">184</span>
<span class="normal">185</span>
<span class="normal">186</span>
<span class="normal">187</span>
<span class="normal">188</span>
<span class="normal">189</span>
<span class="normal">190</span>
<span class="normal">191</span>
<span class="normal">192</span>
<span class="normal">193</span>
<span class="normal">194</span>
<span class="normal">195</span>
<span class="normal">196</span>
<span class="normal">197</span>
<span class="normal">198</span>
<span class="normal">199</span>
<span class="normal">200</span>
<span class="normal">201</span>
<span class="normal">202</span>
<span class="normal">203</span>
<span class="normal">204</span>
<span class="normal">205</span>
<span class="normal">206</span>
<span class="normal">207</span>
<span class="normal">208</span>
<span class="normal">209</span>
<span class="normal">210</span>
<span class="normal">211</span>
<span class="normal">212</span>
<span class="normal">213</span>
<span class="normal">214</span>
<span class="normal">215</span>
<span class="normal">216</span>
<span class="normal">217</span>
<span class="normal">218</span>
<span class="normal">219</span>
<span class="normal">220</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">build_embeddings_matrix</span><span class="p">(</span>
    <span class="n">vocab</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Vocab</span><span class="p">,</span> <span class="n">ChunkVocab</span><span class="p">],</span>
    <span class="n">word_vectors_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
    <span class="n">min_freq</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>  <span class="c1"># pragma: no cover</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Build the embedding matrix using pretrained word vectors.</span>

<span class="sd">    Returns pretrained word embeddings. If a word in our vocabulary is not</span>
<span class="sd">    among the pretrained embeddings it will be assigned the mean pretrained</span>
<span class="sd">    word-embeddings vector</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    vocab: Vocab</span>
<span class="sd">        see `pytorch_widedeep.utils.fastai_utils.Vocab`</span>
<span class="sd">    word_vectors_path: str</span>
<span class="sd">        path to the pretrained word embeddings</span>
<span class="sd">    min_freq: int</span>
<span class="sd">        minimum frequency required for a word to be in the vocabulary</span>
<span class="sd">    verbose: int,  default=1</span>
<span class="sd">        level of verbosity. Set to 0 for no verbosity</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        Pretrained word embeddings</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">word_vectors_path</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2"> not found&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">word_vectors_path</span><span class="p">))</span>
    <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Indexing word vectors...&quot;</span><span class="p">)</span>

    <span class="n">embeddings_index</span> <span class="o">=</span> <span class="p">{}</span>
    <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">word_vectors_path</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">f</span><span class="p">:</span>
        <span class="n">values</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
        <span class="n">word</span> <span class="o">=</span> <span class="n">values</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
        <span class="n">coefs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">values</span><span class="p">[</span><span class="mi">1</span><span class="p">:],</span> <span class="n">dtype</span><span class="o">=</span><span class="s2">&quot;float32&quot;</span><span class="p">)</span>
        <span class="n">embeddings_index</span><span class="p">[</span><span class="n">word</span><span class="p">]</span> <span class="o">=</span> <span class="n">coefs</span>
    <span class="n">f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>

    <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Loaded </span><span class="si">{}</span><span class="s2"> word vectors&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">embeddings_index</span><span class="p">)))</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Preparing embeddings matrix...&quot;</span><span class="p">)</span>

    <span class="n">mean_word_vector</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">embeddings_index</span><span class="o">.</span><span class="n">values</span><span class="p">()),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type]</span>
    <span class="n">embedding_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">embeddings_index</span><span class="o">.</span><span class="n">values</span><span class="p">())[</span><span class="mi">0</span><span class="p">])</span>
    <span class="n">num_words</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">vocab</span><span class="o">.</span><span class="n">itos</span><span class="p">)</span>
    <span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">num_words</span><span class="p">,</span> <span class="n">embedding_dim</span><span class="p">))</span>
    <span class="n">found_words</span> <span class="o">=</span> <span class="mi">0</span>
    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">word</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">vocab</span><span class="o">.</span><span class="n">itos</span><span class="p">):</span>
        <span class="n">embedding_vector</span> <span class="o">=</span> <span class="n">embeddings_index</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">word</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">embedding_vector</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">embedding_matrix</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">embedding_vector</span>
            <span class="n">found_words</span> <span class="o">+=</span> <span class="mi">1</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">embedding_matrix</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">mean_word_vector</span>

    <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span>
            <span class="s2">&quot;</span><span class="si">{}</span><span class="s2"> words in the vocabulary had </span><span class="si">{}</span><span class="s2"> vectors and appear more than </span><span class="si">{}</span><span class="s2"> times&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">found_words</span><span class="p">,</span> <span class="n">word_vectors_path</span><span class="p">,</span> <span class="n">min_freq</span>
            <span class="p">)</span>
        <span class="p">)</span>

    <span class="k">return</span> <span class="n">embedding_matrix</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float32&quot;</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>








  <aside class="md-source-file">
    
    
    
      
  
  <span class="md-source-file__fact">
    <span class="md-icon" title="Contributors">
      
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 4a4 4 0 0 1 4 4 4 4 0 0 1-4 4 4 4 0 0 1-4-4 4 4 0 0 1 4-4m0 10c4.42 0 8 1.79 8 4v2H4v-2c0-2.21 3.58-4 8-4"/></svg>
      
    </span>
    <nav>
      
        <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>
    </nav>
  </span>

    
    
  </aside>





                
              </article>
            </div>
          
          
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
        </div>
        
      </main>
      
        <footer class="md-footer">
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
    <div class="md-copyright__highlight">
      Javier Zaurin and Pavol Mulinka
    </div>
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
        <div class="md-social">
  
    
    
    
    
      
      
    
    <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262m288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    
    
    <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
    
    
      <script src="../../assets/javascripts/bundle.83f73b43.min.js"></script>
      
        <script src="../../stylesheets/extra.js"></script>
      
        <script src="../../javascripts/mathjax.js"></script>
      
        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
      
        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
      
    
  </body>
</html>